package indexer.processer.bxwx

import indexer.processer.UrlPageProcessor
import us.codecraft.webmagic.Site
import us.codecraft.webmagic.selector.Selectable

class BxwxPageProcessor : UrlPageProcessor {

    //    val Log = LoggerFactory.getLogger(javaClass)
    override fun getUrl(): String = "http://www.bxwx9.org/"

    override fun getSite(): Site {
        return Site.me().setRetryTimes(3).setSleepTime(100).setCycleRetryTimes(3).setTimeOut(60_000)
    }

    override fun process(page: us.codecraft.webmagic.Page) {
        val html = page.html
        val urls = html.links().regex("https?://.*\\.bxwx9.org.*").all().filterNot { it.matches(Regex.fromLiteral("https?://.*\\.bxwx9.org/text/.*")) }
        page.addTargetRequests(urls)

        val mainInfo: Selectable = html.xpath("//div[@id='TabCss']")

        if (mainInfo.match()) {
            val name = html.xpath("//div[@id='title']/text()").get()
            val author = html.xpath("//div[@id=info]/a[1]/text()").get()
            val url = page.request.url
            val idx = name.indexOf("章节列表")
            page.putField("name", name.substring(0, idx).trim())
            page.putField("url", url)
            page.putField("author", author)
        } else {
            page.setSkip(true)
        }
    }
}